package org.nerve.webmagic.spider;

import org.junit.Test;
import org.nerve.webmagic.DeepLimitScheduler;
import org.nerve.webmagic.P;
import org.nerve.webmagic.downloader.PhantomJSDownloader2;
import org.nerve.webmagic.processor.BaiduWenkuProcessor;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.utils.UrlUtils;

/**
 * org.nerve.webmagic.spider
 * Created by zengxm on 2016/4/20 0020.
 */
public class BaiduWenkuStarter {

	/**
	 * 采集百度文库
	 *
	 * 结果统计：
	 * 1.不截图，deep=2，耗时 4分24秒（共采集183个页面）
	 * 2.截图，deep=2，耗时 30分钟40秒（共采集187页面，截图187张）
	 */
	@Test
	public void start(){
		Spider.create(
					new BaiduWenkuProcessor()
						.setKeepScreenshotData(false)
				)
				.addUrl(P.BAIKE_BAIDU)
				.thread(3)
				.setScheduler(new DeepLimitScheduler(2))
				.setDownloader(
						new PhantomJSDownloader2()
								.setTakeScreenShot(false)
				)
				.addPipeline(new JsonFilePipeline("/output"))   //位于程序所在盘根目录
				.run();

		System.out.println("-------------------------------------------------------");
	}

	@Test
	public void startWithSimple() throws InterruptedException {
		Spider.create(new PageProcessor() {
			Site site=null;
			@Override
			public void process(Page page) {
				System.out.println(page.getHtml());
			}

			@Override
			public Site getSite() {
				if(site==null){
					site=Site.me()
							.setRetryTimes(3)
							.setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36")
							.setDomain(UrlUtils.getDomain(P.BAIKE_BAIDU))
							.addHeader("Connection","keep-alive")
							.addHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
							.setSleepTime(2000);
				}

				return site;
			}
		})
				.thread(2)
				.setScheduler(new DeepLimitScheduler(1))
				.addUrl(P.BAIKE_BAIDU)
				.run();

		Thread.sleep(60000);
	}
}
